#!C:\WebServer\Python2.4\python.exe
# This script is used to perform hypothesis testing
# Tests available include:
# t-test: difference of 2 means (answer is restricted to those variables that have mean and sd)

import SyntaxParser
import hypothesisSQLGenerator
import conn
import condition
import answer
import patientCondition
import statistic
import Result
import Results
import os
import cgi
print "Content-Type: text/html\n\n"
print "<html><body>"
print """
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

<html>
	<head>
		<title>CDAL Results</title>
		<style type="text/css">
			body {
				font-family: Verdana, Tahoma, Sans-serif;
				font-size: 10pt;
			 	margin-top: 0;
			 	margin-left: 0;
			}
			
			div.title {
			 	float:right;
				font-weight:bold;
				margin-top: 18px;
				clear: left;
				padding-right: 20px;
				color:#FFFFFF;
				background-color: #39638B
			}
			
			#banner {
			 	margin-top: 0;
			 	margin-left: 0;
			 	background-image: url('/images/top_banner_in02.gif');
			 	background-position: 227px;
				background-repeat: no-repeat;
				background-color:#39638B; 
			}
			
			#title {
			 	font-family: Arial, Tahoma, Sans-serif;
			 	margin-top: 10px;
			 	margin-left: 5px;
			 	margin-bottom: 10px;
			 	font-weight: bold;
			 	font-size: 12pt;
			}
			
			#hypothesis_results {
			 	margin-left: 5px;
			 	margin-top: 5px;
			}
			
			#errors, #db_error, #answers_error, #global_time_period_error, #global_location_error, #demo_criterias_error, #prev_diag_error, #curr_diag_error, #med_criterias_error, #cdal_error {
			 	font-size: 8pt;
			 	font-weight: normal;
			 	color: red;
			}
			
			.hypothesis_error {
			 	font-size: 10pt;
			 	font-weight: normal;
			 	margin-left: 5px;
			 	color: red;
			}
			
			#link {
			 	float: right;
			 	margin-right: 5px;
			}
		</style>
	</head>
	
	<body>
		<div id='banner'>
			<div class='title'>
				Clinical Data Mining Project Group
			</div>
		  	<div> 
				<a href='http://www.usyd.edu.au'>
					 <img src='/images/top_banner_in01.gif' width='227' height='52' alt='The University of Sydney' style='border: 0px;' class='decor' /> 
				</a>
			</div>
		</div>
		<div id='link'>
			<a href='cdal.py'>CDAL User Interface</a>
		</div>
		<div id='title'>
			Hypothesis Testing Results
		</div>
"""
# --------- dictionaries for archival database -----------

#path = 'D:\Victor\Elec\Elec4707\Source-Code-v2-3\doc/archive_patientevents_dictionary.txt'
path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/archive_patientevents_dictionary.txt'
archive_patientevents_dictionary = SyntaxParser.buildHash(path)

#path = 'D:\Victor\Elec\Elec4707\Source-Code-v2-3\doc/archive_chartevents_dictionary.txt'
path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/archive_chartevents_dictionary.txt'
archive_chartevents_dictionary = SyntaxParser.buildHash(path)

#path = 'D:\Victor\Elec\Elec4707\Source-Code-v2-3\doc/archive_medevents_dictionary.txt'
path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/archive_medevents_dictionary.txt'
archive_medevents_dictionary = SyntaxParser.buildHash(path)

path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/archive_labevents_dictionary.txt'
archive_labevents_dictionary = SyntaxParser.buildHash(path)

# --------- dictionaries for realtime database -----------

#path = 'D:\Victor\Elec\Elec4707\Source-Code-v2-3\doc/realtime_patientevents_dictionary.txt'
path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/realtime_patientevents_dictionary.txt'
realtime_patientevents_dictionary = SyntaxParser.buildHash(path)

#path = 'D:\Victor\Elec\Elec4707\Source-Code-v2-3\doc/realtime_chartevents_dictionary.txt'
path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/realtime_chartevents_dictionary.txt'
realtime_chartevents_dictionary = SyntaxParser.buildHash(path)

#path = 'D:\Victor\Elec\Elec4707\Source-Code-v2-3\doc/realtime_categoric_terms.txt'
path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/realtime_categoric_terms.txt'
categoric_terms = SyntaxParser.buildCategoricArray(path)

#path = 'D:\Victor\Elec\Elec4707\Source-Code-v2-3\doc/realtime_medevents_dictionary.txt'
path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/realtime_medevents_dictionary.txt'
realtime_medevents_dictionary = SyntaxParser.buildHash(path)

#path = 'D:\Victor\Elec\Elec4707\Source-Code-v2-3\doc/realtime_labevents_dictionary.txt'
path = 'C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\doc/realtime_labevents_dictionary.txt'
realtime_labevents_dictionary = SyntaxParser.buildHash(path)

# --------- definitions of CDAL -----------

time_entities = ["hours", "days", "weeks"]
comparands = [">",">=","<","<=","=","<>"]
types = ["and", "or"]
ref_fns = ["all values", "any value", "last value"] # yet to implement last value
stat_fns = ["mean", "sd", "median", "max", "min", "range", "mode", "sum", "count"]
hypothesis_error_array = []

# ----------default values---------------

default_time_value = 24
default_time_entity = 'hours'
default_location = 'GICU'

# --------- user query -----------

# the answer is restricted to a single variable that have mean and sd (ie. chartevents and labevents),
# But no need to include any ref_fns or stat_fns, since that is done automatically later on in the code

form = cgi.FieldStorage()
test_category = str(form.getvalue("test_category"))
toc = str(form.getvalue("toc"))
db = str(form.getvalue("db"))
answer = str(form.getvalue("answer"))

cond_group1 = str(form.getvalue("med_criterias1"))
if cond_group1 == 'None':
    cond_group1 = ''
    
patient_group1 = str(form.getvalue("demo_criterias1"))
if patient_group1 == 'None':
    patient_group1 = ''
    
time_period_group1 = str(form.getvalue("time_period1"))

if time_period_group1 == 'None': #NO USER INPUT FOR TIME PERIOD
	time_val_group1 = default_time_value #default is used
	time_ent_group1 = default_time_entity 
else:
	#GETTING TIME VALUE AND TIME ENTITY
	index = time_period_group1.lower().strip().find("during the last ", 0)
	if index != 0:
		error = 'Syntax Error in TIME-PERIOD for PATIENT GROUP 1: missing time identifier'
		hypothesis_error_array.append(error)
	else:
		time_period1_ = time_period_group1.strip()[15: len(time_period_group1)]
		if time_period1_.strip() == '':
			error = 'Syntax Error in TIME-PERIOD for PATIENT GROUP 1: missing TIME PERIOD'
			hypothesis_error_array.append(error)
		else:
			time_tokens = time_period1_.split()
	
			if len(time_tokens) != 2:
				error = "Syntax error in TIME-PERIOD for PATIENT GROUP 1: Time should be in the format of [tim_val][tim_ent]"
				hypothesis_error_array.append(error)
			
			else:
				if time_tokens[0].strip().isdigit() != 1:
					error = "Syntax error in TIME-PERIOD for PATIENT GROUP 1: tim_val should be an integer"
					hypothesis_error_array.append(error)
				else:
					time_val_group1 = int(time_tokens[0])
					
				has_time_entity = 0
				for time_entity in time_entities:
					if time_tokens[1].strip().lower() == time_entity.lower():
						has_time_entity = 1
						break
				if has_time_entity == 0:
					error = "Syntax error in TIME-PERIOD for PATIENT GROUP 1: tim_ent should be in ['hours', 'days', 'weeks']"
					hypothesis_error_array.append(error)
				else:
					time_ent_group1 = time_tokens[1]
						
ward_group1 = str(form.getvalue("location1"))

cond_group2 = str(form.getvalue("med_criterias2"))
if cond_group2 == 'None':
    cond_group2 = ''
    
patient_group2 = str(form.getvalue("demo_criterias2"))
if patient_group2 == 'None':
    patient_group2 = ''
    
time_period_group2 = str(form.getvalue("time_period2"))

if time_period_group2 == 'None': #NO USER INPUT FOR TIME PERIOD
	time_val_group2 = default_time_value #default is used
	time_ent_group2 = default_time_entity 
else:
	#GETTING TIME VALUE AND TIME ENTITY
	index = time_period_group2.lower().strip().find("during the last ", 0)
	if index != 0:
		error = 'Syntax Error in TIME-PERIOD for PATIENT GROUP 2: missing time identifier'
		hypothesis_error_array.append(error)
	else:
		time_period2_ = time_period_group2.strip()[15: len(time_period_group2)]
		if time_period2_.strip() == '':
			error = 'Syntax Error in TIME-PERIOD for PATIENT GROUP 2: missing TIME PERIOD'
			hypothesis_error_array.append(error)
		else:
			time_tokens = time_period2_.split()
	
			if len(time_tokens) != 2:
				error = "Syntax error in TIME-PERIOD for PATIENT GROUP 2: Time should be in the format of [tim_val][tim_ent]"
				hypothesis_error_array.append(error)
			
			else:
				if time_tokens[0].strip().isdigit() != 1:
					error = "Syntax error in TIME-PERIOD for PATIENT GROUP 2: tim_val should be an integer"
					hypothesis_error_array.append(error)
				else:
					time_val_group2 = int(time_tokens[0])
					
				has_time_entity = 0
				for time_entity in time_entities:
					if time_tokens[1].strip().lower() == time_entity.lower():
						has_time_entity = 1
						break
				if has_time_entity == 0:
					error = "Syntax error in TIME-PERIOD for PATIENT GROUP 2: tim_ent should be in ['hours', 'days', 'weeks']"
					hypothesis_error_array.append(error)
				else:
					time_ent_group2 = time_tokens[1]
	
ward_group2 = str(form.getvalue("location2"))

ci = str(form.getvalue("ci")) #confidence interval (default = 0.95) 0.975, 0.9, 
test_type = str(form.getvalue("test_type")) #type1: mu1=mu2; type2: mu1>mu2;type3:mu1<mu2 (default=1)

if len(hypothesis_error_array) > 0: # If there are errors
        for hypothesis_error in hypothesis_error_array:
			print "<div class='hypothesis_error'>" + hypothesis_error + "</div>"
			
else:
	
    # --------- Start of all processes -----------

    # for t-test, all that is needed is the mean, sd and size of the variable
    # but in order to obtain the mean and sd, all values are also required
    answer = "all values and mean and sd and count of " + answer

    # initiate variables used in t-test
    mean_group1 = 0
    mean_group2 = 0
    sd_group1 = 1
    sd_group2 = 1
    size_group1 = 1
    size_group2 = 1

    # ----------- Start of realtime database queries for group 1 --------------


    # return the number of hrs of the query (so 1 day = 24 hrs)
    duration_group1 = SyntaxParser.toHours(time_val_group1, time_ent_group1)

    # if the time is <= 48 hrs, then consult realtime db only, o/w need connection to both dbs
    if duration_group1 > 48:
        real_time_val = 48 # use in realtime
    else:
        real_time_val = time_val_group1

    # Obtain the chart-events component of the answer, if any (such as HR, Vent Mode, etc)
    chartevents_answers = SyntaxParser.getAnswers(answer, ref_fns, stat_fns, realtime_chartevents_dictionary)

    # Obtain the lab-events component of the answer, if any (such as calcium, etc)
    labevents_answers = SyntaxParser.getAnswers(answer, ref_fns, stat_fns, realtime_labevents_dictionary)

    # Obtain the patient-events component of the conditions for the archival database, if any (such as mrn, fullname, etc)
    patient_conditions = SyntaxParser.getConditions(patient_group1, comparands, types, realtime_patientevents_dictionary, 'T')
    patientevents_conditions = SyntaxParser.transform_Conditions(patient_conditions, "realtime")

    # Obtain the chart-events component of the conditions, if any (such as HR, Vent Mode, etc)
    chartevents_conditions = SyntaxParser.getConditionArray(cond_group1, comparands, types, realtime_chartevents_dictionary)

    # Obtain the med-events component of the conditions, if any (such as any sedation and inotrope, etc)
    medevents_conditions = SyntaxParser.getConditionArray(cond_group1, comparands, types, realtime_medevents_dictionary)
    
    # Obtain the lab-events component of the conditions, if any (such as calcium, etc)
    labevents_conditions = SyntaxParser.getConditionArray(cond_group1, comparands, types, realtime_labevents_dictionary)
                
    # Generate the SQL Generator object, which is used to generate SQL queries for real-time database
    SQL = hypothesisSQLGenerator.HypSQLGenerator('SNOMED','GICU',None,None,chartevents_answers,None,labevents_answers, patientevents_conditions,chartevents_conditions,medevents_conditions, labevents_conditions, real_time_val, time_ent_group1, ward_group1, categoric_terms)

    resultsArray = [] # array of results
    logicalArray = [] # array of logical operator

    result_set = [] # final set of results

    results_group1 = Results.Results([],[],[]) # initiate results

    patients = None

    #append patients satisfying demographic info to the results array
    if patient_conditions is not None:
        patientQuery = SQL.getPatientEvent()
        patients = conn.toGICU(patientQuery)

    for condition in chartevents_conditions:
        logicalArray.append(condition.getType())
        subQuery = SQL.generateSingleQuery(condition, "chartevents")
        res = conn.toGICU(subQuery)
        resultsArray.append(res)

    for condition in medevents_conditions:
        logicalArray.append(condition.getType())
        subQuery = SQL.generateSingleQuery(condition, "medevents")
        res = conn.toGICU(subQuery)
        resultsArray.append(res)

    for condition in labevents_conditions:
        logicalArray.append(condition.getType())
        subQuery = SQL.generateSingleQuery(condition, "labevents")
        res = conn.toGICU(subQuery)
        resultsArray.append(res)

    # Get the final group of patients whose conditions are satisfied
    finalResultArray = SyntaxParser.getFinalResult(patients, resultsArray, logicalArray)

    # Check that the number of patients satisfying all conditions is not zero, o/w return nothing
    if len(finalResultArray) == 0:
        print "<div class='hypothesis_error'>No result generated...</div>"
    else:
        for ind in range(0,len(chartevents_answers)):
            query_realtime = SQL.generateFinalQuery("chartevents", finalResultArray, ind)

            res_realtime = conn.toGICU(query_realtime)

            # creating a Result Object
            result = Result.Result(chartevents_answers[ind].getVariable(),"chartevents",res_realtime,chartevents_answers[ind].getStat_fn(),chartevents_answers[ind].getRef_fn())

            # Append Result object to Results
            results_group1.AppendRealtime(result)

        for ind in range(0,len(labevents_answers)):
            query_realtime = SQL.generateFinalQuery("labevents", finalResultArray, ind)

            res_realtime = conn.toGICU(query_realtime)
            
            # creating a Result Object
            result = Result.Result(labevents_answers[ind].getVariable(),"labevents",res_realtime,labevents_answers[ind].getStat_fn(),labevents_answers[ind].getRef_fn())

            # Append Result object to Results
            results_group1.AppendRealtime(result)        

    # ----------- End of realtime database queries for group 1 --------------

    # ----------- Start of archival database queries (if required) for group 1 --------------

    if duration_group1 > 48:

        # Obtain the chart-events component of the answer, if any (such as HR, Vent Mode, etc)
        chartevents_answers = SyntaxParser.getAnswers(answer, ref_fns, stat_fns, archive_chartevents_dictionary)

        # Obtain the lab-events component of the answer, if any (such as calcium, etc)
        labevents_answers = SyntaxParser.getAnswers(answer, ref_fns, stat_fns, archive_labevents_dictionary)

        # Obtain the patient-events component of the conditions for the archival database, if any (such as mrn, fullname, etc)
        patient_conditions = SyntaxParser.getConditions(patient_group1, comparands, types, archive_patientevents_dictionary, 'T')
        patientevents_conditions = SyntaxParser.transform_Conditions(patient_conditions, "archive")

        # Obtain the chart-events component of the conditions, if any (such as HR, Vent Mode, etc)
        chartevents_conditions = SyntaxParser.getConditions(cond_group1, comparands, types, archive_chartevents_dictionary)

        # Obtain the med-events component of the conditions, if any (such as any sedation and inotrope, etc)
        medevents_conditions = SyntaxParser.getConditions(cond_group1, comparands, types, archive_medevents_dictionary)

        # Obtain the lab-events component of the conditions, if any (such as calcium, etc)
        labevents_conditions = SyntaxParser.getConditions(cond_group1, comparands, types, archive_labevents_dictionary)

        # Generate the SQL Generator object, which is used to generate SQL queries for both real-time and archival databases
        SQL = hypothesisSQLGenerator.HypSQLGenerator(toc,db,None,None,chartevents_answers,None,labevents_answers, patientevents_conditions,chartevents_conditions,medevents_conditions,labevents_conditions, time_val_group1,time_ent_group1,ward_group1, None)

        result_set = [] # final set of results

        # Here, we split each SQL into 2 halves. The 1st half ('<') contains pid up to 10000, the 2nd ('>=') from 10000 onwards.
        # The rationale behind this is to speed up the overall query
        # There are 4 variables to retrieve for every group_events (mrn, label, value, chart-time)
        # There are only 1 variable to retrieve for all other events (value)
        
        for ind in range(0,len(chartevents_answers)):
            query_archive = SQL.generateArchiveSQL("chartevents", ind, "<") 
            
            res_archive1 = conn.toISM(query_archive) # only 1 variable to retrieve

            query_archive = SQL.generateArchiveSQL("chartevents", ind, ">=")
            
            res_archive2 = conn.toISM(query_archive)

            # combine the results for the 2 sets
            res_archive = SyntaxParser.joinResults(res_archive1, res_archive2)

            # creating a Result Object
            result = Result.Result(chartevents_answers[ind].getVariable(),"chartevents",res_archive,chartevents_answers[ind].getStat_fn(),chartevents_answers[ind].getRef_fn())

            # Append Result object to Results
            results_group1.AppendArchive(result)

        for ind in range(0,len(labevents_answers)):
            query_archive = SQL.generateArchiveSQL("labevents", ind, "<") 
            
            res_archive1 = conn.toISM(query_archive)

            query_archive = SQL.generateArchiveSQL("labevents", ind, ">=")
            
            res_archive2 = conn.toISM(query_archive)

            # combine the results for the 2 sets
            res_archive = SyntaxParser.joinResults(res_archive1, res_archive2)

            # creating a Result Object
            result = Result.Result(labevents_answers[ind].getVariable(),"labevents",res_archive,labevents_answers[ind].getStat_fn(),labevents_answers[ind].getRef_fn())

            # Append Result object to Results
            results_group1.AppendArchive(result)

    # ----------- End of archival database queries for group 1 --------------

    # Combining the realtime and the archival results together to generate final result
    if duration_group1 > 48:
        realtime_result = results_group1.getRealtimeResult()
        archive_result = results_group1.getArchiveResult()
        for real_res in realtime_result:
            real_att = real_res.getAttribute()
            for arch_res in archive_result:
                arch_att = arch_res.getAttribute()

                # some attribute name for realtime and archive db are different, so need to convert them
                if real_att == 'temp. site/value':
                    real_att = 'temperature'
                
                if real_att == arch_att:
                    real_values = real_res.getValues()
                    archive_values = arch_res.getValues()

                    # Join the results from both realtime and archive databases together
                    res_joined = SyntaxParser.joinResults(real_values, archive_values)

                    # creating a Result Object
                    result = Result.Result(real_att, real_res.getEventType(), res_joined, real_res.getStatEnt(), real_res.getRefEnt())

                    # Append Result object to Results
                    results_group1.AppendFinal(result)

                    break
    else:
        results_group1.setFinalResult(results_group1.getRealtimeResult())

    # ----------- Start of realtime database queries for group 2 --------------

    # return the number of hrs of the query (so 1 day = 24 hrs)
    duration_group2 = SyntaxParser.toHours(time_val_group2, time_ent_group2)

    # if the time is <= 48 hrs, then consult realtime db only, o/w need connection to both dbs
    if duration_group2 > 48:
        real_time_val = 48 # use in realtime
    else:
        real_time_val = time_val_group2
        
    # Obtain the chart-events component of the answer, if any (such as HR, Vent Mode, etc)
    chartevents_answers = SyntaxParser.getAnswers(answer, ref_fns, stat_fns, realtime_chartevents_dictionary)

    # Obtain the lab-events component of the answer, if any (such as calcium, etc)
    labevents_answers = SyntaxParser.getAnswers(answer, ref_fns, stat_fns, realtime_labevents_dictionary)

    # Obtain the patient-events component of the conditions for the archival database, if any (such as mrn, fullname, etc)
    patient_conditions = SyntaxParser.getConditions(patient_group2, comparands, types, realtime_patientevents_dictionary, 'T')
    patientevents_conditions = SyntaxParser.transform_Conditions(patient_conditions, "realtime")

    # Obtain the chart-events component of the conditions, if any (such as HR, Vent Mode, etc)
    chartevents_conditions = SyntaxParser.getConditionArray(cond_group2, comparands, types, realtime_chartevents_dictionary)

    # Obtain the med-events component of the conditions, if any (such as any sedation and inotrope, etc)
    medevents_conditions = SyntaxParser.getConditionArray(cond_group2, comparands, types, realtime_medevents_dictionary)

    # Obtain the lab-events component of the conditions, if any (such as calcium, etc)
    labevents_conditions = SyntaxParser.getConditionArray(cond_group2, comparands, types, realtime_labevents_dictionary)
                
    # Generate the SQL Generator object, which is used to generate SQL queries for real-time database
    SQL = hypothesisSQLGenerator.HypSQLGenerator(toc,db,None,None,chartevents_answers,None,labevents_answers, patientevents_conditions,chartevents_conditions,medevents_conditions, labevents_conditions, real_time_val, time_ent_group2, ward_group2, categoric_terms)

    resultsArray = [] # array of results
    logicalArray = [] # array of logical operator

    result_set = [] # final set of results

    results_group2 = Results.Results([],[],[]) # initiate results

    patients = None

    #append patients satisfying demographic info to the results array
    if patient_conditions is not None:
        patientQuery = SQL.getPatientEvent()
        patients = conn.toGICU(patientQuery)

    for condition in chartevents_conditions:
        logicalArray.append(condition.getType())
        subQuery = SQL.generateSingleQuery(condition, "chartevents")
        res = conn.toGICU(subQuery)
        resultsArray.append(res)

    for condition in medevents_conditions:
        logicalArray.append(condition.getType())
        subQuery = SQL.generateSingleQuery(condition, "medevents")
        res = conn.toGICU(subQuery)
        resultsArray.append(res)

    for condition in labevents_conditions:
        logicalArray.append(condition.getType())
        subQuery = SQL.generateSingleQuery(condition, "labevents")
        res = conn.toGICU(subQuery)
        resultsArray.append(res)

    # Get the final group of patients whose conditions are satisfied
    finalResultArray = SyntaxParser.getFinalResult(patients, resultsArray, logicalArray)

    # Check that the number of patients satisfying all conditions is not zero, o/w return nothing
    if len(finalResultArray) == 0:
        print "<div class='hypothesis_error'>No result generated...</div>"
    else:
        for ind in range(0,len(chartevents_answers)):
            query_realtime = SQL.generateFinalQuery("chartevents", finalResultArray, ind)

            res_realtime = conn.toGICU(query_realtime)

            # creating a Result Object
            result = Result.Result(chartevents_answers[ind].getVariable(),"chartevents",res_realtime,chartevents_answers[ind].getStat_fn(),chartevents_answers[ind].getRef_fn())

            # Append Result object to Results
            results_group2.AppendRealtime(result)

        for ind in range(0,len(labevents_answers)):
            query_realtime = SQL.generateFinalQuery("labevents", finalResultArray, ind)

            res_realtime = conn.toGICU(query_realtime)
            
            # creating a Result Object
            result = Result.Result(labevents_answers[ind].getVariable(),"labevents",res_realtime,labevents_answers[ind].getStat_fn(),labevents_answers[ind].getRef_fn())

            # Append Result object to Results
            results_group2.AppendRealtime(result)        

    # ----------- End of realtime database queries for group 2 --------------

    # ----------- Start of archival database queries (if required) for group 2 --------------

    if duration_group2 > 48:

        # Obtain the chart-events component of the answer, if any (such as HR, Vent Mode, etc)
        chartevents_answers = SyntaxParser.getAnswers(answer, ref_fns, stat_fns, archive_chartevents_dictionary)

        # Obtain the lab-events component of the answer, if any (such as calcium, etc)
        labevents_answers = SyntaxParser.getAnswers(answer, ref_fns, stat_fns, archive_labevents_dictionary)

        # Obtain the patient-events component of the conditions for the archival database, if any (such as mrn, fullname, etc)
        patient_conditions = SyntaxParser.getConditions(patient_group2, comparands, types, archive_patientevents_dictionary, 'T')
        patientevents_conditions = SyntaxParser.transform_Conditions(patient_conditions, "archive")
        
        # Obtain the chart-events component of the conditions, if any (such as HR, Vent Mode, etc)
        chartevents_conditions = SyntaxParser.getConditions(cond_group2, comparands, types, archive_chartevents_dictionary)

        # Obtain the med-events component of the conditions, if any (such as any sedation and inotrope, etc)
        medevents_conditions = SyntaxParser.getConditions(cond_group2, comparands, types, archive_medevents_dictionary)

        # Obtain the lab-events component of the conditions, if any (such as calcium, etc)
        labevents_conditions = SyntaxParser.getConditions(cond_group2, comparands, types, archive_labevents_dictionary)

        # Generate the SQL Generator object, which is used to generate SQL queries for both real-time and archival databases
        SQL = hypothesisSQLGenerator.HypSQLGenerator(toc,db,None,None,chartevents_answers,None,labevents_answers, patientevents_conditions,chartevents_conditions,medevents_conditions,labevents_conditions, time_val_group2,time_ent_group2,ward_group2, None)

        result_set = [] # final set of results

        # Here, we split each SQL into 2 halves. The 1st half ('<') contains pid up to 10000, the 2nd ('>=') from 10000 onwards.
        # The rationale behind this is to speed up the overall query
        # There are 4 variables to retrieve for every group_events (mrn, label, value, chart-time)
        # There are only 1 variable to retrieve for all other events (value)
        
        for ind in range(0,len(chartevents_answers)):
            query_archive = SQL.generateArchiveSQL("chartevents", ind, "<") 
            
            res_archive1 = conn.toISM(query_archive) # only 1 variable to retrieve

            query_archive = SQL.generateArchiveSQL("chartevents", ind, ">=")

            res_archive2 = conn.toISM(query_archive)

            # combine the results for the 2 sets
            res_archive = SyntaxParser.joinResults(res_archive1, res_archive2)

            # creating a Result Object
            result = Result.Result(chartevents_answers[ind].getVariable(),"chartevents",res_archive,chartevents_answers[ind].getStat_fn(),chartevents_answers[ind].getRef_fn())

            # Append Result object to Results
            results_group2.AppendArchive(result)

        for ind in range(0,len(labevents_answers)):
            query_archive = SQL.generateArchiveSQL("labevents", ind, "<") 
            
            res_archive1 = conn.toISM(query_archive)

            query_archive = SQL.generateArchiveSQL("labevents", ind, ">=")
            
            res_archive = conn.toISM(query_archive)

            # combine the results for the 2 sets
            res_archive = SyntaxParser.joinResults(res_archive1, res_archive2)

            # creating a Result Object
            result = Result.Result(labevents_answers[ind].getVariable(),"labevents",res_archive,labevents_answers[ind].getStat_fn(),labevents_answers[ind].getRef_fn())

            # Append Result object to Results
            results_group2.AppendArchive(result)

    # ----------- End of archival database queries for group 2 --------------

    # Combining the realtime and the archival results together to generate final result
    if duration_group2 > 48:
        realtime_result = results_group2.getRealtimeResult()
        archive_result = results_group2.getArchiveResult()
        for real_res in realtime_result:
            real_att = real_res.getAttribute()
            for arch_res in archive_result:
                arch_att = arch_res.getAttribute()

                # some attribute name for realtime and archive db are different, so need to convert them
                if real_att == 'temp. site/value':
                    real_att = 'temperature'

                if real_att == arch_att:
                    real_values = real_res.getValues()
                    archive_values = arch_res.getValues()

                    # Join the results from both realtime and archive databases together
                    res_joined = SyntaxParser.joinResults(real_values, archive_values)

                    # creating a Result Object
                    result = Result.Result(real_att, real_res.getEventType(), res_joined, real_res.getStatEnt(), real_res.getRefEnt())

                    # Append Result object to Results
                    results_group2.AppendFinal(result)

                    break
    else:
        results_group2.setFinalResult(results_group2.getRealtimeResult())

    # ----------- End of all processes --------------

    # ----------- Start of t-test hypothesis --------------------

    # show all the reference entity (all values) and statistical entity (mean, sd) for the answer for group 1
    final_result_group1 = results_group1.getFinalResult()
    for res in final_result_group1:
        var = res.getAttribute()
        values = res.getValues()
        values = SyntaxParser.removeNull(values); # discard cases when value = None
        stat_array = res.getStatEnt()
        ref_array = res.getRefEnt()
        if len(values) > 0:
            if stat_array is not None:
                for stat in stat_array:
                    if stat == "mean":
                        mean_group1 = str(statistic.mean(values))
                    elif stat == "sd":
                        sd_group1 = str(statistic.s_d(values))
                    elif stat == "count":
                        size_group1 = str(statistic.count(values))
            '''if ref_array is not None:
                for ref in ref_array:
                    if ref == "all values":
                        print "all values of " + var + ": "
                        for rs in values:
                            print rs'''
        else:
            print "<div class='hypothesis_error'>no value of " + var + " is found.</div>"

    # show all the reference entity (all values) and statistical entity (mean, sd) for the answer for group 2
    final_result_group2 = results_group2.getFinalResult()
    for res in final_result_group2:
        var = res.getAttribute()
        values = res.getValues()
        values = SyntaxParser.removeNull(values);    
        stat_array = res.getStatEnt()
        ref_array = res.getRefEnt()
        if len(values) > 0:
            if stat_array is not None:
                for stat in stat_array:
                    if stat == "mean":
                        mean_group2 = str(statistic.mean(values))
                    elif stat == "sd":
                        sd_group2 = str(statistic.s_d(values))
                    elif stat == "count":
                        size_group2 = str(statistic.count(values))
        else:
            print "<div class='hypothesis_error'>no value of " + var + " is found.</div>"

    # Display results for t-test hypothesis
    print "<div id='hypothesis_results'>"
    print "Two-sample t-test<br/>"
    print "Null Hypothesis: mean of " + var + " for group 1 = mean of " + var + " for group 2<br/>"

    if test_type == "1":
        print "Alternate Hypothesis: mean of " + var + " for group 1 != mean of " + var + " for group 2<br/>"
    elif test_type == "2":
        print "Alternate Hypothesis: mean of " + var + " for group 1 > mean of " + var + " for group 2<br/>"
    elif test_type == "3":
        print "Alternate Hypothesis: mean of " + var + " for group 1 < mean of " + var + " for group 2<br/>"

    # Execute the stand-alone exe file to obtain the result of the hypothesis testing in output.txt
    args = str(mean_group1) + ' ' + str(mean_group2) + ' ' + str(sd_group1) + ' ' + str(sd_group2) + ' ' + str(size_group1) + ' ' + str(size_group2) + ' ' + str(test_type)
    #os.system("C:\Program Files\Apache Group\Apache2\cgi-bin\Source-Code-v2-3\hypothesis_testing\two_sample_t_test " + args)

    #args = str(100) + ' ' + str(102) + ' ' + str(5.8) + ' ' + str(12.4) + ' ' + str(50) + ' ' + str(200) + ' ' + str(1)
    #cmd = "\"hypothesis_testing\\two_sample_t_test.exe\" " + args
    #print cmd
    os.system("\"hypothesis_testing\\two_sample_t_test.exe\" " + args)
    ci = 0.9
    # Parse the result file output.txt to obtain t-statistic and p-value, etc.
    mu1 = SyntaxParser.getHypTestResult('mu1')
    mu2 = SyntaxParser.getHypTestResult('mu2')
    sd1 = SyntaxParser.getHypTestResult('sd1')
    sd2 = SyntaxParser.getHypTestResult('sd2')
    m = SyntaxParser.getHypTestResult('m')
    n = SyntaxParser.getHypTestResult('n')
    df = SyntaxParser.getHypTestResult('df')
    s = SyntaxParser.getHypTestResult('s')
    t = SyntaxParser.getHypTestResult('t')
    pval = SyntaxParser.getHypTestResult('pval')

    print "mean for group 1: " + str(mu1) + "<br/>"
    print "sd for group 1: " + str(sd1) + "<br/>"
    print "size of group 1: " + str(m) + "<br/>"
    print "mean for group 2: " + str(mu2) + "<br/>"
    print "sd for group 2: " + str(sd2) + "<br/>"
    print "size of group 2: " + str(n) + "<br/>"
    print "degree of freedom: " + str(df) + "<br/>"
    print "pooled sd for both groups: " + str(s) + "<br/>"
    print "observed test statistic: " + str(t) + "<br/>"
    print "p-value: " + str(pval) + "<br/>"

    if float(pval) > 1-ci:
        print 'Data consistent with the Null Hypothesis.<br/>'
    else:
        print 'Evidence against the Null Hypothesis in favour with the Alternate Hypothesis.'

    # ----------- End of t-test hypothesis --------------------
    
print "</div></body></html>"
